# coding:utf-8
import json
import re
import thulac
import random


def preprocess_data(inf,stops_path,familyname_path,train_outf,val_outf):
    with open(familyname_path,encoding='utf-8') as f:
        fns = "".join([line.strip() for line in f.readlines()])
        fn = "["+fns+"]"+"[1-9]*某+[甲乙丙丁午己庚辛壬癸]*"\
             +"|["+fns+"]"+"x+[甲乙丙丁午己庚辛壬癸]*"

    with open(stops_path,encoding='utf-8') as f:
        stops = [line.strip() for line in f.readlines()]

    train_out = open(train_outf,encoding='utf-8',mode='w')
    val_out = open(val_outf, encoding='utf-8', mode='w')
    with open(inf,encoding='utf-8') as f:
        lines = f.readlines()
        random.shuffle(lines)
        for i,line in enumerate(lines):
            line = json.loads(line)
            fact = line['fact']
            try:
                fact = replace_stop_info(fact,fn)
            except Exception as e:
                print(e)
            relevant_articles = line['meta']['relevant_articles']
            if i<int(len(lines)*0.9):
                out = train_out
            else:
                out = val_out
            for ra in relevant_articles:
                out.write(str(ra)+'\t')
            out.write(fact.strip()+'\n')
            out.flush()
            print(i)



def replace_stop_info(fact,fn):
    thu = thulac.thulac(seg_only=True)
    fact = re.sub('[0-9]{4}年[0-9]{1,2}月[0-9]{1,2}日[0-9]{1,2}时许'
                      '|[0-9]{4}年[0-9]{1,2}月[0-9]{1,2}日.*?[0-9]{1,2}时许'
                      '|[0-9]{4}年[0-9]{1,2}月[0-9]{1,2}日.*?午'
                      '|[0-9]{4}年[0-9]{1,2}月[0-9]{1,2}日晚上'
                      '|[0-9]{4}年[0-9]{1,2}月[0-9]{1,2}日晚'
                      '|[0-9]{4}年[0-9]{1,2}月[0-9]{1,2}日凌晨'
                      '|[0-9]{4}年[0-9]{1,2}月[0-9]{1,2}日'
                      '|[0-9]{4}年[0-9]{1,2}月份到[0-9]{1,2}月份'
                      '|[0-9]{4}年[0-9]{1,2}月'
                      '|[0-9]{4}年', '<TIME>', fact)
    # 替换所有的18岁以下年龄为<CHILD>标识,替换所有的18岁以上为<AUDLT>标识
    fact = re.sub(r'(?<!\d)(1[8-9]|[2-9]\d|[1-9]\d\d)岁', '<AUDLT>', fact)
    fact = re.sub(r'(?<!\d)([1-9]|1[1-7]|未满.*?)岁', '<CHILD>', fact)

    # 替换金额
    fact = re.sub(r'(?<!\d)(\d|[1-9]\d|[1-9]\d\d)(\.\d{0,2}){0,1}余?元', '<ONE>', fact)
    fact = re.sub(r'(?<!\d)(1\d\d\d)(\.\d{0,2}){0,1}余?元', '<TWO>', fact)
    fact = re.sub(r'(?<!\d)(2\d\d\d)(\.\d{0,2}){0,1}余?元', '<THREE>', fact)
    fact = re.sub(r'(?<!\d)([3-4]\d\d\d)(\.\d{0,2}){0,1}余?元', '<FOUR>', fact)
    fact = re.sub(r'(?<!\d)([5-9]\d\d\d)(\.\d{0,2}){0,1}余?元', '<FIVE>', fact)
    fact = re.sub(r'(?<!\d)(1\d\d\d\d)(\.\d{0,2}){0,1}余?元', '<SIX>', fact)
    fact = re.sub(r'(?<!\d)(1|一)(\.\d{0,2}){0,1}余?万余?元', '<SIX>', fact)
    fact = re.sub(r'(?<!\d)([2-4]\d\d\d\d)(\.\d{0,2}){0,1}余?元', '<SEVEN>', fact)
    fact = re.sub(r'(?<!\d)([2-4]|[二两三四])(\.\d{0,2}){0,1}余?万余?元', '<SEVEN>', fact)
    fact = re.sub(r'(?<!\d)(5\d\d\d\d)(\.\d{0,2}){0,1}余?元', '<EIGHT>', fact)
    fact = re.sub(r'(?<!\d)(5|五)(\.\d{0,2}){0,1}余?万余?元', '<EIGHT>', fact)

    fact = re.sub(r'(?<!\d)([6-9]\d\d\d\d|[1-9]\d\d\d\d\d)(\.\d{0,2}){0,1}余?元', '<NINE>', fact)
    fact = re.sub(r'(?<!\d)([6-9]|[六七八九])(\.\d{0,2}){0,1}余?万余?元', '<NINE>', fact)
    fact = re.sub(r'(?<![二三四五六七八九])十余?万余?元', '<NINE>', fact)
    fact = re.sub(r'(?<![二三四五六七八九])十[一二三四五六七八九]余?万余?元', '<NINE>', fact)
    fact = re.sub(r'(?<!\d)([1-9]\d)(\.\d{0,2}){0,1}余?万余?元', '<NINE>', fact)
    fact = re.sub(r'(?<!\d)([二三四五六七八九])(\.\d{0,2}){0,1}十余?万余?元', '<NINE>', fact)
    fact = re.sub(r'(?<!\d)([二三四五六七八九])(\.\d{0,2}){0,1}十[一二三四五六七八九]余?万余?元', '<NINE>', fact)
    # 替换人名
    fact = re.sub(fn, '<PERSON>', fact)
    fact = re.sub(r'[\s+\.\!\/_,$%^*(+\"\')]+|[+——()?【】“”！，。？、~@#￥%……&*（）《》；：×X]+', "", fact)
    # 去除停用词
    # for st in stops:
    #     fact = fact.replace(st,'')
    fact = thu.cut(fact, text=True).replace(' <', '').replace('<', '').replace('> ', '').replace('>', '')
    return fact




if __name__ == '__main__':

    preprocess_data('./datasets/gamedatas/train.json','./datasets/gamedatas/stopwords.txt','./datasets/gamedatas/familyname.txt'
                    ,'./datasets/gamedatas/train.txt','./datasets/gamedatas/val.txt')
